import asyncio
import datetime
import json
import logging
from os import makedirs
from os.path import exists

from pyppeteer import launch
from pyquery import PyQuery as pq

RESULTS_DIR='results'

exists(RESULTS_DIR) or makedirs(RESULTS_DIR)
WINDOW_WIDTH, WINDOW_HEIGHT=1366, 768
HEADLESS=False


# 使用asyncio库的异步执行 详见：https://kaiwu.lagou.com/course/courseInfo.htm?courseId=46#/detail/pc?id=1677
# pyquery用法详见 https://www.cnblogs.com/gj5379/p/8514535.html
# Python 异步 IO(asyncio)、多进程(multiprocessing)、多线程(multithreading)性能对比 https://www.jianshu.com/p/cac56b3d9a18

class cloudTencentCuxiao(object):
    async def init(self):
        global browser, page
        browser=await launch(headless=HEADLESS, ignoreHTTPSErrors=True,
                             args=['--disable-infobars',
                                   f'--window-size={WINDOW_WIDTH},{WINDOW_HEIGHT}'])
        page=await browser.newPage()
        await page.setViewport({'width': WINDOW_WIDTH, 'height': WINDOW_HEIGHT})

    async def scrape_page(self):
        await page.goto('https://cloud.tencent.com/product')
        # 防止检测(即隐藏WebDriver模仿成普通浏览器请求)
        await page.evaluateOnNewDocument('Object.defineProperty(navigator, "webdriver", {get: () => undefined})')
        await page.waitForSelector('.po-card')  # Page 对象调用 waitForSelector 方法，传入选择器，如果加载出来了，立即返回，否则会持续等待直到超时
        doc=pq(await page.content())

        # 1.列表形式
        product_name_list=[item.text() for item in doc('.po-tit').items()]  # 获取商品名称
        product_href_list=[item.attr('href') for item in
                          doc('.po-card a:nth-child(1)').items()]  # 获取商品链接，.po-card 下第一个a标签里的href属性的值
        # print('ProductName:', productName, 'ProductHref:', ProductHref)

        # 2.文本形式
        for item in doc('.po-card').items():
            productName=item('.po-tit').text()
            ProductHref=item('a:nth-child(1)').attr('href')
            print(productName, ProductHref)
        await browser.close()
        return product_name_list, product_href_list  # 只返回json格式的准备保存

    async def save_data(self, data):
        data_path=f'{RESULTS_DIR}/data.json'
        json.dump(data, open(data_path, 'w', encoding='utf-8'), ensure_ascii=False, indent=2)

    async def main(self):
        await self.init()
        try:
            data=await self.scrape_page()
            await self.save_data(data)
        finally:
            logging.info('失败，清检查原因')
            await browser.close()


if __name__=='__main__':
    start_time=datetime.datetime.now()
    asyncio.get_event_loop().run_until_complete(cloudTencentCuxiao().main())
    # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy(main.py())) # uvloop只有类Unix系统才能使用
    end_time=datetime.datetime.now()
    print("运行时间：{}s".format((end_time-start_time).seconds))
